﻿using CsharpHttpHelper;
using CsharpHttpHelper.Item;
using NestClient;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

namespace httpLibrary
{
    public class NBClass
    {
        public static Mutex mutex = new Mutex();
        public static List<String> listOldString = new List<string>();
        public void Start(List<string> url)
        {
            ThreadPool.SetMaxThreads(15, 15);

            HtmlContextObj content = new HtmlContextObj();
            content.Id = url[0];
            content.Url = url[0];
            content.ParentId = "";
            content.Flag = 0;

            //MyNestClient.Intance.AddIndex<HtmlContextObj>("xsy_db");

            MyNestClient.Intance.InsertDataToIndex(content, "xsy_db");

            ThreadPool.QueueUserWorkItem(ThreadMain,null);

            //ThreadPool.QueueUserWorkItem(TaskMethod, url);
        }

        public void SecondStart()
        {
            ThreadPool.SetMaxThreads(15, 15);

            ThreadPool.QueueUserWorkItem(ThreadMain, null);
        }

        public static Mutex ThreadCountMutex = new Mutex();
        public static int ThreadCount = 0;
        private void ThreadMain(object e)
        {
            while(true)
            {
                if (ThreadCount<1)
                {
                    Dictionary<string, object> idc = new Dictionary<string, object>();
                    idc.Add("flag",0);
                    List<HtmlContextObj> rs = MyNestClient.Intance.SearchData<HtmlContextObj>(idc);
                    if(rs !=null)
                    {
                        ThreadCountMutex.WaitOne();
                        ThreadCount += rs.Count;
                        ThreadCountMutex.ReleaseMutex();

                        foreach(HtmlContextObj obj in rs)
                        {
                            ThreadPool.QueueUserWorkItem(TaskMethod, obj);
                        }
                    }
                }

                Thread.Sleep(100);
            }
        }

        private bool isIndexed(HtmlContextObj content,string indexName)
        {
            bool rs = false;
            mutex.WaitOne();
            rs = MyNestClient.Intance.SearchDataExists<HtmlContextObj>(content, indexName);
            mutex.ReleaseMutex();
            return rs;
        }
        public void TaskMethod(object obj)
        {
            try
            {
                HtmlContextObj url = obj as HtmlContextObj;
                if (url != null && string.IsNullOrWhiteSpace(url.Url) == false)
                {
                    string htmlString = GetHtml(url.Url);

                    if (string.IsNullOrWhiteSpace(htmlString) == false)
                    {
                        if (url.SetHtmlDate(htmlString))
                        {
                            
                            try
                            {
                                MyNestClient.Intance.InsertDataToIndex(url, "xsy_db");
                            }
                            catch
                            {
                            }
                        }

                        //获取页面所有的a标签的url
                        List<string> newlist = GetAHREFToList(htmlString, url.Url);
                        foreach (string str1 in newlist)
                        {
                            HtmlContextObj content1 = new HtmlContextObj();
                            content1.Id = str1;
                            content1.Url = str1;
                            content1.ParentId = url.Url;
                            content1.Flag = 0;
                            if (isIndexed(content1, "xsy_db") == false)
                            {
                                MyNestClient.Intance.InsertDataToIndex(content1, "xsy_db");
                            }
                        }
                    }
                    else
                    {
                        url.Flag = 1;
                        try
                        {
                            MyNestClient.Intance.InsertDataToIndex(url, "xsy_db");
                        }
                        catch
                        {
                        }
                    }
                }
            }
            catch (Exception ex)
            {
            }
            finally
            {
                ThreadCountMutex.WaitOne();
                ThreadCount --;
                ThreadCountMutex.ReleaseMutex();
            }
        }

        private string GetHtml(string url)
        {
            HttpHelper http = new HttpHelper();
            HttpItem item = new HttpItem()
            {
                Method="HEAD",
                URL = url,
                Timeout = 5000,
                ReadWriteTimeout=5000,
                Allowautoredirect = true,
                //ProxyIp = "192.168.0.64:8087",
                Encoding = null,
            };
            //得到HTML代码

            HttpResult result = http.GetHtml(item);
            if (result.StatusCode == System.Net.HttpStatusCode.OK  && result.Header != null)
            {
                string Vary1 = result.Header["Content-Type"];
                if (Vary1!=null &&Vary1.Contains("text") )
                {
                    item.Method = "GET";
                    result = http.GetHtml(item);

                    if ( result.Html.Contains("</html>") || result.Html.Contains("</HTML>"))
                    {
                        return result.Html;
                    }
                }
            }
            return "";
        }

        private List<string> GetAHREFToList(string htmlStr,string url)
        {
            List<string> rs = new List<string>();
            List<AItem> newlist = HttpHelper.GetAList(htmlStr);
            if (newlist!= null)
            {
                foreach (AItem m in newlist)
                {
                    if (m != null && string.IsNullOrWhiteSpace(m.Href) == false)
                    {
                        string url1 = m.Href.Replace("\"", "");//得到href值    
                        if (string.IsNullOrWhiteSpace(url1) == false && rs.Contains(url1) == false)
                        {
                            if (url1.IndexOf("http:") != 0)
                            {
                                try
                                {
                                    url1 = url.Substring(0, url.IndexOf('/', 6)) + url1;
                                }
                                catch { }
                            }
                            if (NBClass.IsUrl(url1))
                            {
                                //if(url1.Contains("edu")||url1.Contains(""))
                                rs.Add(url1);
                            }
                        }
                    }
                }
            }

            return rs;
        }

        /// <summary>
        /// 判断一个字符串是否为url
        /// </summary>
        /// <param name="str"></param>
        /// <returns></returns>
        public static bool IsUrl(string str)
        {
            try
            {
                string Url = @"^http(s)?://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?$";
                return Regex.IsMatch(str, Url);
            }
            catch (Exception ex)
            {
                return false;
            }
        }
    }
}
